package com.nlp.visualization.crawler.processor;

import com.google.gson.Gson;
import com.nlp.visualization.common.CONSTANTS;
import com.nlp.visualization.pojo.weibo.WeiboList;
import com.nlp.visualization.pojo.weibo.WeiboUserDetail;
import com.nlp.visualization.pojo.weibo.WeiboUserFollow;
import com.nlp.visualization.pojo.weibo.WeiboUserInfo;
import org.apache.commons.lang.StringUtils;
import org.openqa.selenium.Cookie;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author LXM
 * @Title: ProcessorFlowerAndFollow
 * @Description: 微博的粉丝以及关注
 * @date 2018/3/18下午2:14
 */
public class ProcessorWeiboFlowerAndFollow implements PageProcessor {

    private final String DOMAIN = "m.weibo.cn";

    //用来存储cookie信息
    private Set<Cookie> cookies;


    //判断当前页面属于什么类型
    private final String USER_INFO_PAGE = "retcode=6102&type=uid";
    private final String FOLLOW_LIST_PAGE = "_followers_";
    private final String WEIBO_USER_DETAIL_PAGE = "_-_INFO&title";

    //微博关注的URL的模板
    private final String WEIBO_LIST_URL = "https://m.weibo.cn/api/container/getIndex?containerid={oid}_-_followers_-_{uid}&luicode=10000011&lfid={fid}&page={page}";

    //第二个模板
    private final String WEIBO_LIST_URL_2 = "https://m.weibo.cn/api/container/getIndex?containerid={cid}&luicode=10000011&lfid={fid}&page={page}";

    private final String WEIBO_USER_DETAIL_URL = "https://m.weibo.cn/api/container/getIndex?containerid={oid}_-_INFO&title";


    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me()
            .setRetryTimes(CONSTANTS.CRAWLER_RETRY_TIME)
            .setSleepTime(CONSTANTS.CRAWLER_RETRY_WAITING_TIME)
            .setDomain(DOMAIN)
            .setRetrySleepTime(5)
            .setSleepTime(3000)
            .setUserAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1")
            .setCharset("UTF-8");

    Logger logger = LoggerFactory.getLogger(ProcessorWeiboFlowerAndFollow.class);

    private boolean IsFirstTime = true;//是否是第一次爬这个页面

    public void initProcessor() {
        IsFirstTime = true;
    }


    @Override
    public void process(Page page) {
        //根据不同的页面，按不同的形式保存
        String url = page.getUrl().toString();
        if (url.indexOf(USER_INFO_PAGE) != -1 && IsFirstTime) {
            IsFirstTime = false;
            //第一次爬取页面，需要将他的关注都页面爬取下来
            doSaveUserInfo(page);
        } else if (url.indexOf(USER_INFO_PAGE) != -1 && !IsFirstTime) {
            //第二次爬取页面，只需要把那些人都主页资料爬取下来
            doSecondTimeCrewl(page);
        } else if (url.indexOf(WEIBO_USER_DETAIL_PAGE) != -1) {
            doSaveUserInfoDetail(page);
        } else if (url.indexOf(FOLLOW_LIST_PAGE) != -1) {
            doSaveFollow(page);

        } else {
            page.setSkip(true);
        }
    }

    /**
     * 第二种爬取页面的情况，只需要把资料爬取下来保存即可
     *
     * @param page
     */
    private void doSecondTimeCrewl(Page page) {

        try {
            logger.info("保存用户普通资料页");
            //用来解析json
            Gson gson = new Gson();
            //部分二：保存从页面上爬取的信息
            WeiboUserInfo userInfo = gson.fromJson(unicodeToString(page.getJson().toString().replaceAll("\\$","")), WeiboUserInfo.class);
            //取出用户的主页信息，并将要访问的主页的链接添加到target中
            String containerId = "";
            List<WeiboUserInfo.DataBean.TabsInfoBean.TabsBean> tabsBeans = userInfo.getData().getTabsInfo().getTabs();
            for (WeiboUserInfo.DataBean.TabsInfoBean.TabsBean tab : tabsBeans) {
                if (tab.getTitle().equals("主页"))
                    containerId = tab.getContainerid();
            }
            page.putField("userinfo", userInfo);
            String userWeiboInfoDetail = WEIBO_USER_DETAIL_URL.replace("{oid}", containerId);
            //增加用户的微博详情页，如果info页不为空的清空下
            if (!StringUtils.isBlank(containerId))
                page.addTargetRequest(new Request(userWeiboInfoDetail));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 保存用户关注的列表
     *
     * @param page
     */
    private void doSaveFollow(Page page) {
        try {
            //用来解析json
            Gson gson = new Gson();
            WeiboUserFollow weiboUserFollow = gson.fromJson(page.getJson().toString(), WeiboUserFollow.class);
            //取出总微博数量
            Long total = Long.parseLong(weiboUserFollow.getData().getCardlistInfo().getTotal() + "");
            //总页数
            int totalPage = (int) (total / 20 + 1);
            //生成Request对象
            Request[] requests = new Request[totalPage];
            int count = 1;
            //当前的URL
            String currentURL = page.getUrl().toString().split("&page")[0];
            for (Request re : requests) {
                ++count;
                re = new Request(
                        currentURL + "&page=" + count);
                page.addTargetRequest(re);
            }
            //遍历粉丝的资料
            for (WeiboUserFollow.DataBean.CardsBean.CardGroupBean groupBean : weiboUserFollow.getData().getCards().get(0).getCard_group()) {
                String request = "https://m.weibo.cn/api/container/getIndex?display=0&retcode=6102&type=uid&value=" + groupBean.getUser().getId();
                page.addTargetRequest(new Request(request));
            }
            page.putField("follow", weiboUserFollow);
            logger.info("保存用户1个用户关注页面成功");

        } catch (Exception e) {
            logger.error("爬取页面异常");
            e.printStackTrace();
        }

    }

    /**
     * 根据用户的主页，获取用户关注的人的地址
     *
     * @param page
     */
    private void doSaveUserInfo(Page page) {
        try {
            //用来解析json
            Gson gson = new Gson();
            //部分二：保存从页面上爬取的信息
            WeiboUserInfo userInfo = gson.fromJson(unicodeToString(page.getJson().toString()), WeiboUserInfo.class);
            //取出用户的主页信息，并将要访问的主页的链接添加到target中
            String fid = userInfo.getData().getTabsInfo().getTabs().get(1).getContainerid();
            String userId = new String((userInfo.getData().getUserInfo().getId() + "").getBytes("UTF-8"), "UTF-8");
            String oid = userInfo.getData().getFans_scheme().split("_-_fansrecomm_-")[0];
            oid = oid.substring(oid.length() - 6, oid.length());
            String userWeiboList = WEIBO_LIST_URL.replace("{fid}", fid).replace("{uid}", userId).replace("{page}", "1").replace("{oid}", oid);
            //增加用户关注列表到target端
            page.addTargetRequest(new Request(userWeiboList));
        } catch (Exception e) {
            logger.error("爬取异常");
            e.printStackTrace();
        }
        logger.info("保存用户普通资料页");


    }

    /**
     * 获取用户信息详情页的数据
     *
     * @param page
     */
    private void doSaveUserInfoDetail(Page page) {
        try {
            logger.info("保存更多资料详情页");
            //用来解析json
            Gson gson = new Gson();
            //部分二：保存从页面上爬取的信息
            WeiboUserDetail weiboUserDetail = gson.fromJson(unicodeToString(page.getRawText().toString()), WeiboUserDetail.class);
            //取出用户的主页信息，并将要访问的主页的链接添加到target中
            page.putField("userDetail", weiboUserDetail);

        } catch (Exception e) {
            e.printStackTrace();
        }

    }


    @Override
    public Site getSite() {
        //将获取到的cookie信息添加到webmagic中
        if (cookies != null)
            for (Cookie cookie : cookies) {
                site.addCookie(cookie.getName().toString(), cookie.getValue().toString());
            }
        return site;
    }

    public static String unicodeToString(String str) {

        Pattern pattern = Pattern.compile("(\\\\u(\\p{XDigit}{4}))");
        Matcher matcher = pattern.matcher(str);
        char ch;
        while (matcher.find()) {
            //group 6728
            String group = matcher.group(2);
            //ch:'木' 26408
            ch = (char) Integer.parseInt(group, 16);
            //group1 \u6728
            String group1 = matcher.group(1);
            str = str.replace(group1, ch + "");
        }
        return str;
    }
}
